package spark.example

import org.apache.spark.rdd.RDD
import org.apache.spark.sql.SaveMode._
import org.apache.spark.sql.types._
import org.apache.spark.sql.{Row, SQLContext}

case class ParquetConnSettings(SQLContext: SQLContext) {

  val parquetSchema = new StructType(
  Array(
    StructField("msgType", StringType, false),
    StructField("did", StringType, false),
    StructField("gid", StringType, false),
    StructField("ip", StringType, false),
    StructField("thirdCloudId",StringType, false),
    StructField("registered", StringType, false),
    StructField("as",  MapType(StringType, StringType, true), true)))

    val c2c_post_schema = StructType(
      Array(
        StructField("msgType", StringType),
        StructField("did", StringType),
        StructField("gid", StringType, false),
        StructField("ip", StringType, false),
        StructField("thirdCloudId",StringType, false),
        StructField("registered", StringType, false)
      )
    )

  val device_Schema = new StructType()
    .add("registered","string").add("ip","string").add("msgType","string")
    .add("gid","string").add("did","string").add("thirdCloudId","string")

  def save(rdd: RDD[Row]): Unit = {
    val df = SQLContext.createDataFrame(rdd, device_Schema)
    df.show
    df.printSchema()
    df.write.mode(Append).parquet("hdfs://cdhmaster:8020/tmp/sparktest/")
  }
}
